import pymysql
from lxml import etree
import requests
import json
import re
import time
from walmart.字段 import words
import random
import threading
from walmart.动态ip import ips
conn = pymysql.connect(host='gz-cdb-ezvn7csl.sql.tencentcdb.com', port=58892, user='root', password='shengye2021',
                       database='bailun_bltpro', charset='utf8')
cursor = conn.cursor()


def maga():
    headers = {
        'Host': 'www.walmart.com',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
        'sec-ch-ua-mobile': '?0',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Service-Worker-Navigation-Preload': 'true',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-User': '?1',
        'Sec-Fetch-Dest': 'document',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
        'Cookie': 'AID=wmlspartner%253D0%253Areflectorid%253D0000000000000000000000%253Alastupd%253D1637050244515; ACID=e57a8b4c-df53-4682-85a9-a0f6cf417ed1; hasACID=true; locDataV3=eyJpbnRlbnQiOiJTSElQUElORyIsInBpY2t1cCI6W3siYnVJZCI6IjAiLCJub2RlSWQiOiIzMDgxIiwiZGlzcGxheU5hbWUiOiJTYWNyYW1lbnRvIFN1cGVyY2VudGVyIiwibm9kZVR5cGUiOiJTVE9SRSIsImFkZHJlc3MiOnsicG9zdGFsQ29kZSI6Ijk1ODI5IiwiYWRkcmVzc0xpbmUxIjoiODkxNSBHZXJiZXIgUm9hZCIsImNpdHkiOiJTYWNyYW1lbnRvIiwic3RhdGUiOiJDQSIsImNvdW50cnkiOiJVUyIsInBvc3RhbENvZGU5IjoiOTU4MjktMDAwMCJ9LCJnZW9Qb2ludCI6eyJsYXRpdHVkZSI6MzguNDgyNjc3LCJsb25naXR1ZGUiOi0xMjEuMzY5MDI2fSwiaXNHbGFzc0VuYWJsZWQiOnRydWUsInNjaGVkdWxlZEVuYWJsZWQiOnRydWUsInVuU2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZX1dLCJzaGlwcGluZ0FkZHJlc3MiOnsibGF0aXR1ZGUiOjM4LjQ4MjY3NywibG9uZ2l0dWRlIjotMTIxLjM2OTAyNiwicG9zdGFsQ29kZSI6Ijk1ODI5IiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeUNvZGUiOiJVUyIsImxvY2F0aW9uQWNjdXJhY3kiOiJsb3ciLCJnaWZ0QWRkcmVzcyI6ZmFsc2V9LCJhc3NvcnRtZW50Ijp7Im5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJhY2Nlc3NQb2ludHMiOm51bGwsImludGVudCI6IlBJQ0tVUCIsInNjaGVkdWxlRW5hYmxlZCI6ZmFsc2V9LCJpbnN0b3JlIjpmYWxzZSwicmVmcmVzaEF0IjoxNjM3MDUzODQ0NjQyLCJ2YWxpZGF0ZUtleSI6InByb2Q6djI6ZTU3YThiNGMtZGY1My00NjgyLTg1YTktYTBmNmNmNDE3ZWQxIn0%3D; assortmentStoreId=3081; hasLocData=1; locGuestData=eyJpbnRlbnQiOiJTSElQUElORyIsInN0b3JlSW50ZW50IjoiUElDS1VQIiwibWVyZ2VGbGFnIjpmYWxzZSwicGlja3VwIjp7Im5vZGVJZCI6IjMwODEiLCJ0aW1lc3RhbXAiOjE2MzcwNTAyNDQ2MzJ9LCJwb3N0YWxDb2RlIjp7InRpbWVzdGFtcCI6MTYzNzA1MDI0NDYzMiwiYmFzZSI6Ijk1ODI5In0sInZhbGlkYXRlS2V5IjoicHJvZDp2MjplNTdhOGI0Yy1kZjUzLTQ2ODItODVhOS1hMGY2Y2Y0MTdlZDEifQ%3D%3D; TB_Latency_Tracker_100=1; TB_Navigation_Preload_01=1; TB_DC_Flap_Test=1; vtc=YeDTYcBVVJrmBJn6nfVFmc; bstc=YeDTYcBVVJrmBJn6nfVFmc; mobileweb=0; xpa=CfaEu; exp-ck=CfaEu1; _pxhd=/31pu0tzO1ECcUVm0TNn55u-08C8rpNRCZvDOnL0MqLMAibINQJeQKaDDvcYPc1t4aVGFFOHJF2ht5J3fmevnA==:cmMqj7pGW6pBtipT-FCLoSRKW8QFZbs9jtK6utKFrbUui5K5oAfhXrjVRb0Gv436XRrw0DmPDSSvuxbDsDiF5CJo-n81taZQaeSbCSP0gNI=; TBV=7; adblocked=false; pxcts=bbc3ed41-46b4-11ec-8697-fdbb46d83dd9; _pxvid=b2839e3b-46b4-11ec-a6e3-585a4b527450; ak_bmsc=8AABA27234D5386498DC3DA580A8A827~000000000000000000000000000000~YAAQDtrIF1iSSw99AQAAaZ/MJw2QCZ7NqEcAk4OJmEFCvIipXAXmk7y7iPC0L1lsWk26hecPTR2SEHacE+1LXGJ7rfnSCLIXHhT+2z+yeoJZF1krANdZEJVfxc9a8TjPMmLkCZscgR2z9s8qTYJkeLy6lxxUlD2lpiNQGJ9KCtTzZUXZBN+/2HbUBPHFLqlTjklgJ28KpzDsnzxG7owlkGObkXsPSNaTv2AGmmGBqoitQOzgr/OVE6kiUT2Qpmxlm6AfzWdMqXqYD0I8Dq1a3I3TkQasvK4TLEeBmLAunBjVD5sW4t4jC+5uf7iOj8vA3rD8t7/OmYBXxG6oSxFEKuGYK76ijpVNnWBiztTWJv+YjofLYls375KaeFGSwoixTnDjlB495XpZFxXDe9HoMNhbEKSLoeFDIf/CbpjwNB9bQNUxi1tEMl2n9rEO5C63LAis3U9NbMGfovsCePa6NUL8Jjcupa+En4AfTTA=; xpm=1%2B1637050244%2BYeDTYcBVVJrmBJn6nfVFmc~%2B0; tb_sw_supported=true; TB_SFOU-100=1; akavpau_p2=1637052508~id=91997cd6658b0a7566794db6c542e699; com.wm.reflector="reflectorid:0000000000000000000000@lastupd:1637051925000@firstcreate:1637050244515"; xptwg=2896669889:16834EFF5CF7C80:3B109BE:72E878EE:7F11AD15:5663913B:; TS01b0be75=01538efd7c81328bbb6eff6dd23390172201515443bf6c134fce8f64ddccda398e68c29d974899c599e59edd6d84c2e538cc979fe9; TS013ed49a=01538efd7c81328bbb6eff6dd23390172201515443bf6c134fce8f64ddccda398e68c29d974899c599e59edd6d84c2e538cc979fe9; bm_sv=A4501CB648113980B59BAAE1B05D5CC2~xkRIXprLft9ygnxDqx/fxqRsYOkmsY24QtxriB8SWuX63le4ZJuCCQc4ycCyIEoh5P1HcK9fGMia+XyzKMZX507WFn/A3hR/pyK14taRLCNkYiSd9lxXZB6qw3kXLZoxXLgO56HIv8bUXDfjRoWnVeziSypn4b/Qa+wAMGTjaz8=',

    }
    list1 = []
    for uurl in words():
        uurl = uurl.strip()
        # print(uurl)
        xiancheng = threading.Thread(target=all_xpath, args=(uurl, headers,))
        list1.append(xiancheng)
    for li1 in list1:
        li1.start()
    for li2 in list1:
        li2.join()


def mysql(SalePrice, Title, Specifications, VariationSpecific, CategoryPath, Images, Description, itmeid):
    CrawlerTime = time.strftime('%Y/%m/%d %H:%M:%S')
    # sql = 'updata walmart_lunhui set SalePrice=%s,Title=%s,Specifications=%s,VariationSpecific=%s,CategoryPath=%s,Images=%s,CrawlerTime=%s,Description=%s values(SalePrice,Title,Specifications,VariationSpecific,CategoryPath,Images,CrawlerTime,Description)'
    cursor.execute(
        "update t_bi_walmart_listing_crawler set SalePrice=%s,Title=%s,Specifications=%s,VariationSpecific=%s,CategoryPath=%s,Images=%s,CrawlerTime=%s,Description=%s where ItemId=%s",
        [SalePrice, Title, Specifications, VariationSpecific, CategoryPath, Images, CrawlerTime, Description, itmeid])

    conn.commit()
    print(SalePrice, '\n', Title, '\n', Specifications, '\n', VariationSpecific, '\n', CategoryPath, '\n',
          Images, '\n', Description, '\n', itmeid)


def all_xpath(uurl, headers):
    print(uurl)
    # url = 'http://www.walmart.com/ip/Women-s-Electric-USB-Winter-Heated-Vest-Warm-Slim-Heating-Coat-Jackets-Sleeveless-Padded-Machine-Washable-Double-Button-Control-3-Temperature-Gear/495824768'
    response = requests.get(uurl, headers=headers, verify=False,proxies=ips()).content.decode()
    # print('response1', response)
    res = etree.HTML(response)

    src = res.xpath('//img/@src')  # 图片链接
    # print(src,type(src))
    print('rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr')
    str0 = ''
    for i in src:
        if i[:5] == 'https' and '?odnHeight=80&odnWidth=80&odnBg=FFFFFF' in i:
            str0 += '|' + i + '\n'
        # print(i)
    str2 = str0.replace('?odnHeight=80&odnWidth=80&odnBg=FFFFFF', '')
    # print(str2)
    color_size = res.xpath('//div[@class="w-100 f6 mt2"]/div[@class="mb2 mid-gray"]/span')
    print(color_size)
    for i in color_size:
        print(i.text)
    try:
        color_lei = color_size[0].text
        color = color_size[1].text  # 颜色
    except Exception as g:
        color_lei = None
        color = None
    try:
        size1_lei = color_size[2].text
        size1 = color_size[3].text  # 尺寸
    except Exception as g:
        size1_lei = None
        size1 = None
    # print(color, size1)

    load = res.xpath('//script')
    # print(load[4].text,type(load[4]))
    str1 = ''
    for a in json.loads(load[4].text)["itemListElement"]:
        # print(a["item"]["name"])
        str1 += a["item"]["name"] + '/'
    # print(str1)  # 路径
    title = res.xpath('//h1[@class="f3 b lh-copy dark-gray mt1 mb2"]')[0].text
    # print(title)  # 标题
    price = res.xpath('//span[@itemprop="price"]')[0].text
    # print(price)  # 价格

    description = json.loads(res.xpath('//script')[3].text)
    # print(description["description"])
    try:
        Description = re.findall('Description:(.*?)</', description["description"], re.S)[0].replace('<br>',
                                                                                                     '').replace(
            '&nbsp;', '')
        # print(Description)  # 描述
    except Exception as g:
        Description = re.findall('Specifications:(.*)<br><br>', description["description"], re.S)[0].replace('<br>',
                                                                                                             '').replace(
            '&nbsp;', '').replace('</h2>', '').replace('<h2>', '').replace('<b>', '').replace('</b>', '')
        # print(Description)  # 描述
    # itmeid = re.findall('\d+\.?\d*', url, re.S)[0]
    itmeid = uurl.split('/')[-1]
    # print(itmeid)

    # print('*'*30)
    for e in load:
        # print(e.text)

        if 'specifications' in str(e.text):
            shuxing2 = json.loads(re.findall('specifications":(.*?),"warnings', e.text)[0])
            # print(shuxing2)
            SalePrice = price
            Title = title
            Specifications = shuxing2

            if color == None:

                VariationSpecific = [
                    {
                        "name": f"{size1_lei}",
                        "value": f"{size1}"
                    }
                ]
            elif size1 == None:
                VariationSpecific = [
                    {
                        "name": f"{color_lei}",
                        "value": f"{color}"
                    }
                ]
            else:
                VariationSpecific = [
                    {
                        "name": f"{color_lei}",
                        "value": f"{color}"
                    },
                    {
                        "name": f"{size1_lei}",
                        "value": f"{size1}"
                    }
                ]
            CategoryPath = str1[:-1]
            Images = str2[1:]
            SalePrice = str(SalePrice)
            Title = str(Title)
            Specifications = str(json.dumps(Specifications))
            VariationSpecific = str(json.dumps(VariationSpecific))
            CategoryPath = str(CategoryPath)
            Images = str(Images)
            Description = str(Description)
            itmeid = f"{itmeid}".strip()
            # mysql(SalePrice, Title, Specifications, VariationSpecific, CategoryPath,
            #       Images, Description, itmeid)


if __name__ == '__main__':
    maga()
